import json
import requests
import re
import pymysql


connection = pymysql.connect(
    host='localhost',
    user='root',
    password='123456',
    database='mydb1',
    charset='utf8mb4',
    cursorclass=pymysql.cursors.DictCursor
)


session = requests.Session()
data = {
    "csrf_token": "JuUZjLcpehvHAlFEIzXgPKnOxCsQWfytMDkYiRSNGrVBomTadbqw",
    "username": "123",
    "password": "123"
}

login_url = "https://quotes.toscrape.com/login"

res = session.post(login_url, data=data)
print(res.cookies)

all_data = []

url = "https://quotes.toscrape.com/"
while True:
    response = session.get(url)
    html_content = response.text

    # 定义正则表达式来匹配名言、作者和标签
    quote_pattern = re.compile(r'<span class="text" itemprop="text">(.*?)</span>')
    author_pattern = re.compile(r'<small class="author" itemprop="author">(.*?)</small>')
    tag_pattern = re.compile(r'<a class="tag" href="/tag/.*?">(.*?)</a>')

    # 查找所有名言
    quotes = quote_pattern.findall(html_content)
    authors = author_pattern.findall(html_content)
    tag_matches = tag_pattern.findall(html_content)

    tag_index = 0
    for i in range(len(quotes)):
        text = quotes[i]
        author = authors[i]
        tags = []
        while True:
            # 根据页面结构判断标签是否属于当前名言
            if tag_index < len(tag_matches) and f'<span class="text" itemprop="text">{text}</span>' in html_content.split(tag_matches[tag_index])[0]:
                tags.append(tag_matches[tag_index])
                tag_index += 1
            else:
                break
        all_data.append({
            'text': text,
            'author': author,
            'tags': tags
        })

    # 查找下一页的链接


for data in all_data:
    print(data)

with open('quotes.json', 'w') as f:
    json.dump(all_data, f, ensure_ascii=False, indent=4)

# 将数据插入到 MySQL 数据库中
try:
    with connection.cursor() as cursor:
        for data in all_data:
            text = data['text']
            author = data['author']
            tags = ','.join(data['tags'])
            insert_query = "INSERT INTO quotes (text, author, tags) VALUES (%s, %s, %s)"
            cursor.execute(insert_query, (text, author, tags))
    connection.commit()
    print("数据已成功插入到数据库中。")
except pymysql.Error as e:
    print(f"数据插入出错: {e}")
finally:
    connection.close()